package main

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"os"
	"spider/pkg/spider"
	"time"
)

func getMD5Hash(text string) string {
	hasher := md5.New()
	hasher.Write([]byte(text))
	return hex.EncodeToString(hasher.Sum(nil))
}

func main() {

	timeout := flag.Uint("t", 5, "Request timeout in seconds")
	maxDepth := flag.Uint("d", 3, "Maximum spider depth")
	maxUrls := flag.Uint("n", 500, "Maximum number of urls can be visited")
	saveHtml := flag.Bool("s", false, "Save html to runtime directory")

	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("Start Url expected")
	}

	mySpider := spider.NewSpider(context.Background())
	mySpider.RequestTimeout = time.Duration(*timeout) * time.Second
	mySpider.MaxDepth = int(*maxDepth)
	mySpider.MaxVisitedUrls = int(*maxUrls)
	if *saveHtml {

		if _, err := os.Stat("runtime"); os.IsNotExist(err) {
			err = os.Mkdir("runtime", 0777)
			if err != nil {
				log.Fatal("Can not create runtime directory")
			}
		}

		mySpider.PageHandler = func(ctx context.Context, link string, depth int, text string) {
			err := os.WriteFile(fmt.Sprintf("runtime/test-%s.txt", getMD5Hash(link)), []byte(text), 0644)
			if err != nil {
				log.Println(err)
			}
		}

	}

	startUrl := flag.Arg(0)
	mySpider.Start(startUrl)
}
